{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G42lpMKJ7xgv"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def normal_pdf(x, mu, sigma):\n",
        "    \"\"\"\n",
        "    Computes the PDF of a normal distribution with mean mu and standard deviation sigma\n",
        "    at point x.\n",
        "    \"\"\"\n",
        "    return (1 / (np.sqrt(2 * np.pi) * sigma)) * np.exp(-((x - mu) ** 2) / (2 * sigma ** 2))\n",
        "\n",
        "# Example usage:\n",
        "x_values = np.linspace(-5, 5, 100)\n",
        "mu = 0\n",
        "sigma = 1\n",
        "pdf_values = normal_pdf(x_values, mu, sigma)\n",
        "\n",
        "plt.plot(x_values, pdf_values)\n",
        "plt.title(\"Normal PDF with mean = 0 and std dev = 1\")\n",
        "plt.xlabel(\"x\")\n",
        "plt.ylabel(\"PDF(x)\")\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from scipy.stats import norm\n",
        "\n",
        "# Generate some data from a normal distribution\n",
        "mu = 5\n",
        "sigma = 2\n",
        "data = np.random.normal(mu, sigma, 1000)\n",
        "\n",
        "# Estimate the mean and standard deviation of the distribution from the data\n",
        "mu_hat, sigma_hat = norm.fit(data)\n",
        "\n",
        "# Plot a histogram of the data with the estimated distribution overlaid\n",
        "plt.hist(data, bins=50, density=True, alpha=0.5)\n",
        "x = np.linspace(mu - 4*sigma, mu + 4*sigma, 100)\n",
        "plt.plot(x, norm.pdf(x, mu_hat, sigma_hat), 'r-', lw=2)\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "F6gRPUbF8bme"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from scipy.stats import norm\n",
        "\n",
        "# Generate a dataset\n",
        "data = np.random.normal(5, 2, 1000)\n",
        "\n",
        "# Define the log-likelihood function\n",
        "def log_likelihood(theta, x):\n",
        "    mu, sigma = theta\n",
        "    return np.sum(norm.logpdf(x, loc=mu, scale=sigma))\n",
        "\n",
        "# Define the range of mu and sigma values to evaluate the likelihood over\n",
        "mus = np.linspace(0, 10, 100)\n",
        "sigmas = np.linspace(0.1, 5, 100)\n",
        "\n",
        "# Create a grid of mu and sigma values\n",
        "mu_grid, sigma_grid = np.meshgrid(mus, sigmas)\n",
        "\n",
        "# Calculate the log-likelihood over the grid of mu and sigma values\n",
        "ll_grid = np.zeros_like(mu_grid)\n",
        "for i in range(mu_grid.shape[0]):\n",
        "    for j in range(mu_grid.shape[1]):\n",
        "        ll_grid[i,j] = log_likelihood([mu_grid[i,j], sigma_grid[i,j]], data)\n",
        "\n",
        "# Plot the log-likelihood surface\n",
        "fig = plt.figure(figsize=(8,6))\n",
        "ax = fig.add_subplot(111, projection='3d')\n",
        "ax.plot_surface(mu_grid, sigma_grid, ll_grid, cmap='viridis')\n",
        "ax.set_xlabel('mu')\n",
        "ax.set_ylabel('sigma')\n",
        "ax.set_zlabel('log likelihood')\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "r-vZII7z8jQG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "def kl_divergence(p, q):\n",
        "    \"\"\"\n",
        "    Computes KL divergence between two probability distributions p and q.\n",
        "    \"\"\"\n",
        "    p = np.asarray(p, dtype=np.float)\n",
        "    q = np.asarray(q, dtype=np.float)\n",
        "    return np.sum(np.where(p != 0, p * np.log(p / q), 0))\n",
        "\n",
        "# Example usage:\n",
        "p = [0.2, 0.3, 0.5]\n",
        "q = [0.25, 0.25, 0.5]\n",
        "kl_div = kl_divergence(p, q)\n",
        "print(\"KL divergence:\", kl_div)\n"
      ],
      "metadata": {
        "id": "9BSwyHWv9B3Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.datasets import load_digits\n",
        "from sklearn.mixture import GaussianMixture\n",
        "\n",
        "# Load MNIST dataset\n",
        "digits = load_digits()\n",
        "X, y = digits.data, digits.target\n",
        "\n",
        "# Fit GMM to the data\n",
        "n_components = 20\n",
        "gmm = GaussianMixture(n_components=n_components, covariance_type='full', random_state=42)\n",
        "gmm.fit(X)\n",
        "\n",
        "# Generate new samples from the GMM\n",
        "n_samples = 100\n",
        "samples, _ = gmm.sample(n_samples)\n",
        "\n",
        "# Reshape the samples into images\n",
        "samples = np.reshape(samples, [n_samples, 8, 8])\n",
        "\n",
        "# Visualize the generated samples\n",
        "fig, axes = plt.subplots(10, 10, figsize=(10, 10))\n",
        "for i in range(10):\n",
        "    for j in range(10):\n",
        "        axes[i][j].imshow(samples[i * 10 + j], cmap='gray')\n",
        "        axes[i][j].axis('off')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "M78J_4Y59G_-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import tensorflow_probability as tfp\n",
        "\n",
        "# Load MNIST dataset\n",
        "(x_train, y_train), _ = tf.keras.datasets.mnist.load_data()\n",
        "x_train = x_train.astype('float32') / 255.\n",
        "x_train = x_train.reshape(x_train.shape[0], -1)\n",
        "\n",
        "# Define model\n",
        "num_components = 100\n",
        "num_features = x_train.shape[1]\n",
        "batch_size = 1000\n",
        "gmm = tfp.distributions.MixtureSameFamily(\n",
        "    mixture_distribution=tfp.distributions.Categorical(\n",
        "        probs=tf.ones([num_components], dtype=tf.float32) / num_components),\n",
        "    components_distribution=tfp.distributions.MultivariateNormalDiag(\n",
        "        loc=tf.Variable(tf.zeros([num_components, num_features], dtype=tf.float32)),\n",
        "        scale_diag=tfp.util.TransformedVariable(tf.ones([num_components, num_features], dtype=tf.float32),\n",
        "                                                bijector=tfb.Softplus())\n",
        "    )\n",
        ")\n",
        "\n",
        "# Define loss function\n",
        "@tf.function\n",
        "def nll_loss(x):\n",
        "    return -tf.reduce_mean(gmm.log_prob(x))\n",
        "\n",
        "# Define optimizer\n",
        "optimizer = tf.optimizers.Adam(learning_rate=0.001)\n",
        "\n",
        "# Train model\n",
        "num_epochs = 100\n",
        "num_batches = x_train.shape[0] // batch_size\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "    epoch_loss = 0.\n",
        "    for batch in range(num_batches):\n",
        "        x_batch = x_train[batch*batch_size:(batch+1)*batch_size]\n",
        "        with tf.GradientTape() as tape:\n",
        "            loss = nll_loss(x_batch)\n",
        "        gradients = tape.gradient(loss, gmm.trainable_variables)\n",
        "        optimizer.apply_gradients(zip(gradients, gmm.trainable_variables))\n",
        "        epoch_loss += loss\n",
        "    print('Epoch: {}, Loss: {}'.format(epoch, epoch_loss / num_batches))\n",
        "\n",
        "# Generate samples from the trained model\n",
        "num_samples = 10\n",
        "samples = gmm.sample(num_samples)\n",
        "samples = tf.reshape(samples, [num_samples, 28, 28])\n",
        "samples = tf.clip_by_value(samples, 0, 1)\n"
      ],
      "metadata": {
        "id": "zffkk-f29nya"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}